package com.bdqn.spark.chapter05.broadcast

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

object Spark02_Broadcast {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[4]").setAppName("broadcast")
    val sc = new SparkContext(sparkConf)

    val rdd1: RDD[(String, Int)] = sc.makeRDD(List(
      ("a", 1), ("b", 2), ("c", 3)
    ))

    // 把一部分的数据保存起来
    val dataMap = mutable.Map(
      ("a", 4), ("b", 5), ("c", 6)
    )

    // 通过map操作，可以实现同样的效果，避免了shuffle操作
    val mapRDD: RDD[(String, (Int, Int))] = rdd1.map {
      case (k, v) => {
        val num: Int = dataMap.getOrElse(k, 0)
        (k, (v, num))
      }
    }
    mapRDD.collect().foreach(println)

    sc.stop()
  }
}
